{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resampling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Figure3-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T15:37:58.464560Z",
     "start_time": "2019-04-01T15:37:57.538Z"
    }
   },
   "outputs": [],
   "source": [
    "library(ggplot2)\n",
    "# set the path\n",
    "setwd(getwd())\n",
    "# load data\n",
    "# 这里的数据和书上做图用的数据有些不同\n",
    "session_times <- read.csv(\"../psds_data/web_page_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T15:39:19.615887Z",
     "start_time": "2019-04-01T15:39:19.365Z"
    }
   },
   "outputs": [
    {
     "data": {},
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAHgCAMAAABOyeNrAAACu1BMVEUAAAABAQEEBAQGBgYH\nBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQXFxcYGBgZGRkaGhob\nGxscHBwdHR0eHh4fHx8gICAiIiIkJCQlJSUmJiYoKCgqKiorKyssLCwtLS0uLi4wMDAzMzM0\nNDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NERERHR0dI\nSEhKSkpLS0tMTExNTU1OTk5PT09QUFBRUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpb\nW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxt\nbW1vb29wcHBxcXF0dHR1dXV2dnZ3d3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGC\ngoKDg4OEhISFhYWGhoaHh4eIiIiJiYmKioqLi4uMjIyNjY2Pj4+RkZGSkpKTk5OUlJSVlZWW\nlpaXl5eYmJiZmZmampqbm5ucnJydnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eo\nqKipqamqqqqrq6usrKytra2urq6vr6+wsLCxsbGysrK0tLS1tbW2tra3t7e4uLi6urq7u7u8\nvLy9vb2+vr6/v7/AwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fJycnKysrLy8vMzMzNzc3Ozs7P\nz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh\n4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq6urr6+vs7Ozt7e3u7u7v7+/w8PDx8fHy8vLz\n8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9/f3+/v7////q4OvXAAAACXBIWXMAABJ0\nAAASdAHeZh94AAARkklEQVR4nO3d/Z9U1X3A8duUaBBijW0isW3aWJuxqba6PiRp7AAK41pc\ng0hRkrZIFigEhfhEanwoVVArIdRtUnyIVqqom6CxSloiRSFFzBZEJLTIw8KysO75M3rv7OwB\ngT0737v3O+ec9fP+gZ0Zlu/93vv6vJjLykpiAAWJ7wUwPBEWVBAWVBAWVBAWVBAWVBAWVBAW\nVIjDOrg7OAc+2Od7BR96Dvve4CTyh3VgV3AOmL2+V/Cht8f3BidBWPEjLG2EFQ7Cih9haSOs\ncBBW/AhLG2GFg7DiR1jaCCschBU/wtJGWOEgrPgRljbCCgdhxY+wtBFWOAgrfoSljbDCQVjx\nIyxtH9Gwmiq+NzgJwoofYWkjrHAQVvwISxthhYOw4kdY2ggrHPnDOtQVnCOm2/cKPjRVfG9w\nogP5wzq4JzhdptP3Cj40VXxvcBL5w+KtMBTD7K2QsEJBWNoIKxzDK6zDhBWKYRTWtu98qXT5\nIzt9r9F4hKXrplJmse81Go+wVK2pdlU6v8P3Ig1HWKqW9YVVWuN7kYYjLFU/qIX1M9+LNBxh\nqdrcVO3qyl/5XqThCEvXkxekXV32qu81Go+wlG1YuuiRX/pewgPC0sZX3sNBWPEjLG2EFQ7C\nih9haSOscBBW/AhLG2GFg7DiR1jaCCschBU/wtJGWOEgrPgRljbCCgdhxY+wtBFWOAgrfoSl\njbDCQVjxIyxthBWOYRXWqtkfve/92kVY+h4oPet7BR8ISxthhYOw4kdY2ggrHIQVP8LSRljh\nIKz4EZY2wgoHYcWPsLQRVjgIK36EpY2wwkFY8SMsbYQVDsKKH2FpI6xwEFb8Yg9r96JJ1967\nj7CCE3lYvXO++V//ccMiwgpO5GHtKG8x5qXxPYQVmsjDeutvjxizflwXYYUm8rBSve8vusU+\nIaxQxB/WvHJL9Z+93539a0hL6/gFDfZwaY3vFXxoqvje4ET2jqmusHb94u4pB9KP+7+eevpw\ncB4stftewYemiu8NTnSo/rB2bslCnPhK/3PeCkMR+Vvh6pb0t7eu8a8SVmgiD2t38z2bNiyY\neoCwQhN5WGbjrErLHe/ap4QVitjDOg5hhYKwtBFWOAgrfoSljbDCQVjxIyxthBUOwoofYWkj\nrHAQVvwISxthhYOw4kdY2ggrHIQVP8LSRljhIKz4EZY2wgoHYcWPsLQRVjgIK36EpY2wwkFY\n8SMsbYQVDsKKH2FpI6xwEFb8CEsbYYWDsOJHWNoIKxyEFT/C0kZY4SCs+BGWNsIKB2HFj7C0\nEVY4CCt+hKWNsMJBWPEjLG2EFQ7Cih9haSOscBBW/AhLG2GFg7DiR1jaCCschBU/wtJGWOEg\nrPgRljbCCgdhxY+wtBFWOPKHdehgcJaWXvS9gg9NFd8bnKgzf1hd+4OztPS87xV8aKr43uBE\n+/KHxVthKIbZWyFhhYKwtBFWOAgrfoSljbDCQVjxIyxthBUOwoofYWkjrHAQVvwISxthhYOw\n4kdY2ggrHIQVP8LSRljhIKyGeOu2+XrOv1Rx+F3v5DtjwmqIfypFK+clJayGWF5a8maU7iit\nynfGhNUQy0tPSa9vGB4krF2EpYCwMoRVOMLKEFbhCCtDWIUjrAxhFY6wMoRVOMLKEFbhCCtD\nWIUjrAxhFY6wMoRVOMLKEFbhCCtDWIUjrAxhFY6wMoRVOMLKEFbhCCtDWIUjrAxhFY6wMoRV\nOMLKEFbhCCtDWIUjrAxhFY6wMoRVOMLKEFbhCCtDWIUjrAxhFY6wMoRVOMLKEFbhCCtDWIUr\nPqy9z35/e+cRxyEJS4Cw+i0+NUnaV5/ZNvAhCUuAsGqeSC56JGnfdknyb0df2794cvPCd+xT\nwhIgrJo/Oaf7/aTdHPl809HXvjNt7Rs3T7b/2C9hCRBWzScWmCws0zravtRZfs2YgxPb+58T\nlgBh1YyZ0xfWjLPsS1tn7jOm95on+p8TlgBh1VQ+szsL650zxn/4OC+VN/Y/JCwBwqr55eiz\n5yWtrZ88ZdOxr/Y8Oe7vs497x6baeoLzUOknvlcYwMp4w1qT64QP2wnHfbnhja8mqYvWHvta\nx99Unu7NHhCWFGEdtX/d2j0femH9lbfuPuYpb4UCvBUO7PC1S3qPfU5YAoRVs3XiWWf0sS+t\nLb/wWuq9/ueEJUBYNX+WXDjtxir70o/KVf/a/5ywBAirZuSNxx/hBIQlQFg1v7Nk0EMSlgBh\n1dzU1DPYIQlLgLBqus8777vLqgY8JGEJEFbNyo8lNQMekrAECKvmC2c99YvNVQMekrAECKtm\n1H2DHpKwBAir5kvfHvSQhCVAWDVrx7Qff4jjEZYAYdWM+73kt86pGvCQhCVAWDWXWwMekrAE\nCKt+hCVAWPUjLAHCyh6NNqZkDXhIwhIgrNSZnzPmK9aAhyQsAcKqH2EJEFbq7LvqOiRhCRBW\n9mheXYckLAHCMoSlgbAMYWkgrOxR063HGPCQhCVAWNmjDxnwkIQlQFjZo8n/fowBD0lYAoRl\nuMfSQFhGO6zl8/VcU/q64vScV7jvrAlLO6xLSrGamu+EqwgrNfWJkx3hBDnDunjcO1F6uzQl\n3wlXEVb98oY1odATb5huwhKxEwjLjbBk7ATCciMsGTuBsNwIS8ZOICw3wpKxEwjLjbBk7ATC\nciMsGTuBsNwIS8ZOICw3wpKxEwjLjbBk7ATCciMsGTuBsNwIS8ZOICw3wpKxEwjLjbBk7ATC\nciMsGTuBsNwIS8ZOICw3wpKxEwjLjbBk7ATCciMsGTuBsNwIS8ZOICw3wpKxEwjLjbBk7ATC\nciMsGTtBHFbX/lziDWtqvhOuaos3rNW5TnifnSAPqzOXiMPKd8JVEYf1fK4T3m8n8FboNtS3\nwjtXR2kO91jKhhpWtAhLF2HJ2AtHWG6EJWMvHGG5DTWsJW9G6XbCUsbXsWTsBMJyIywZO4Gw\n3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw\n3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw\n3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw\n3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw\n3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw\n3AhLxk4gLDfCkrETCMuNsGTshHrC6pm0++gTwhIgLJfu5WXCyoewHJ6+okxYORGWw56ONYSV\nE2E5ba6Ftbcl9diRXC6JNqxp+U64amW8Yf0k1wl32wmSsHZn/1bw0nybxhvWDUP41Y/HG9bL\nuX5dj30kCauKt0IB3gqdCCsvwnIqJKzxe6O0i7BE7IRGhdVUihVhSdgJhDUYwpKwExr13woJ\nKyrxhHXxl78XpX8kLBE7gb/d4MafCmXsBMJyIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJy\nIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJyIywZO4Gw3AhLxk4gLDfCkrETCMuNsGTsBMJy\nG2pYd66O0hzCUja0sNp8/2Wy/J7Ld8b2whGW29DC2v7ocj0XfEVx+A925jtje+EIy21oYalq\nqvje4CTshSMsN8KSsReOsNwIS8ZeOMJyIywZe+EIy42wZOyFIyw3wpKxF46w3AhLxl44wnIj\nLBl74QjLjbBk7IUjLDfCkrEXjrDcCEvGXjjCciMsGXvhGhbWBS1RmkRYIg0P63rffw0kt9uK\nvfLFISxtD5Se9b2CD4SljbDCQVjxIyxthBUOwoofYWkjrHAQVvwISxthhYOw4kdY2ggrHIQV\nP8LSRljhIKz4EZY2wgoHYcWPsLQRVjgIK37DLKyDe4KzpPSc7xV8aKr43uAk8od1qCs4D5Ze\n9L2CD00V3xuc6ED+sHgrDMUweyskrFAQljbCCgdhxY+wtBFWOAgrfoSljbDCQVjx6+3xvcFJ\nEFb8CEsbYYWDsOJHWNoIKxyEFT/C0kZY4SCs+BGWNsIKB2HFj7C0EVY4CCt+hKWNsMJBWPEj\nLG2EFQ7Cih9haSOscBBW/AhLG2GFg7DiR1jaCCschBU/wtJGWOEgrPgRlrZVs3/mewUfCEvb\nAbPX9wo+EJY2wgoHYcWPsLQRVjgIK36EpY2wwkFY8SMsbYQVDsKKH2FpI6xwEFb8CEsbYYWD\nsOJHWNoIKxyEFT/C0kZY4SCs+BGWNsIKx7AK66cPbfC9gg/Lfuh7g5PIH1aAHi6t8b2CD00V\n3xu4EFa0CEsbYQWIsKJFWNq69x7xvYIP+zp9b+AyHMJCgAgLKggLKuIK685yasqSg4N93q/K\nE7sasU+D1Hfa09NPGv+NFxqz0qAiC6t106YNT1XuG+zzHp9QHk5/UKzvtKffvWnTuqXl/2zM\nToOJLKyF2Y8rmnsH+bwZd828owHrNEp9pz39e+kPvTP+oREbDS7GsJ644gOzbX5zZc4WY3Z9\n++qZGyodpmvJ9ZUF22qftq38yj9fecDjngWr77SrYZk5y/3teazIwlrQ09O9ccrNxtw05+c/\nnz3T9NywcOMr08Z2mNtbX994+7X7+z6tbeKhLeUXvW5aqPpOe/qynp7OF5q3eF62JrKwsrvY\n8owdpvex7ca0TzIvX5X+xvR8uWPbuD3GHGl5tfpZvdMXmd4pCz3vWqD6Tnt69bOWD3ab0CCR\nhTVr8+bNO7NL17PxmXtbJpm2Wenj7eWOl8vNqbE/qn7WW9md+wPj93tdtUj1nfb0ezZv/u+X\npg36J5vGiCys/t+GDs35xoq1qyeZFbPTJzvKHWuu3pnpa2lZeey4cWPLq72tWbT6TrvvHuun\n5X2etvywSMNaO+GwMc9MMj9uPpi+N5Q73i5vNabznq3Zz/VOubkj9bX5HhctVl2nXQvr5fKg\nX+VriEjDWl9+cuO/tFyxufv6Wze9+lfjtvZ+66/XrV847XD2c2+UX8s+LM/uP4aHuk67+nWs\njWumBnJvGWlYvSuumXTnjllzzbvfam7dMnaf6bz3a8237qj+3JLrerIP75ZX+VqzaHWddt/N\ne8s9YbwTRhbWCXa/kt7Svj0xkD8JNUwEpx17WBMe3/U/s+73vUajRXDakYdl1s2cMPm+YfRF\n9jqFf9qxh4VAERZUEBZUEBZUEBZUEBZUEJbAyiRz2hfbQv7KZCAIS2Bl8ufz5s297vTku743\nCR9hCaxMlmUftn1yZNBfmwwCYQnUwjI3Jus8bxI+whLoD2tu8px59I9Gf+LcxdnN1qqLR53/\nw7tGpI86ms8e+adPel0xGIQl0B/WpUnHY0nptm+em6wwpu3X/nDB9FP+OA3rjdGfnn3z55NA\nvv/KM8ISqIbV++685KLe8b950JiukdeZrrNK6aNnkjSsr579f8Z0X3TqXt97hoCwBPq+3JAk\nv7vJ7EobMv97arP5cfJo+qj3D0aY/cnc91OPJM/63jMEhCVQ/XLDvFtWZt+8sOXR1ss+njSb\nh/tu5K8aYdbVsku+73nNIBCWQP89Vmrxx8687qHXxzSb+/vCunqEWZu0tldt97dhOAhL4GhY\nnb/+lx+kb4BnNJvVSVv2whdGmPeT7Lv9zLbVQf+f9hqFsASOhrU+uTv9sS2pmH2fOi+9eX8u\nu3m/8PT096ojl575kfwfVx6PsASOhnXoM6dNvuXy0z/7G/enN1nnzp8++rLTjHntlE/NbD2n\nejMPwhI45h7r9S+PGnN1xwufvSp99YujLmmf+7n0xTfHfnrUhcPmm86GhrCG6MiO6i3VX1zo\ne5HAENYQHRxxffrje6fe4nuRwBDWUM1IWpbd+9sj3/O9R2AIa6gO/93vf3zM2Ld8rxEawoIK\nwoIKwoIKwoIKwoIKwoIKwoIKwoIKwoIKwoKK/wcgLdYiLPz3zQAAAABJRU5ErkJggg==",
      "text/plain": [
       "plot without title"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "options(repr.plot.width=5, repr.plot.height=4)  # plot size\n",
    "ggplot(session_times, aes(x=Page, y=Time)) + geom_boxplot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Figure3-4(Permutation Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T15:54:02.629526Z",
     "start_time": "2019-04-01T15:54:02.594Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "0.356666666666666"
      ],
      "text/latex": [
       "0.356666666666666"
      ],
      "text/markdown": [
       "0.356666666666666"
      ],
      "text/plain": [
       "[1] 0.3566667"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mean_a <- mean(session_times[session_times['Page']=='Page A', 'Time'])\n",
    "mean_b <- mean(session_times[session_times['Page']=='Page B', 'Time'])\n",
    "mean_b - mean_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T15:46:01.042839Z",
     "start_time": "2019-04-01T15:46:01.017Z"
    }
   },
   "outputs": [],
   "source": [
    "perm_func <- function(x, n1, n2){\n",
    "    n <- n1 + n2\n",
    "    idx_b <- sample(1:n, n1)\n",
    "    idx_a <- setdiff(1:n, idx_b)\n",
    "    mean_diff <- mean(x[idx_b]) - mean(x[idx_a])\n",
    "    return(mean_diff)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注意这里，巧妙地使用索引与集合的差运算实现了不放回抽样。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T15:50:35.613903Z",
     "start_time": "2019-04-01T15:50:35.340Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAHgCAMAAABOyeNrAAAC/VBMVEUAAAABAQECAgIDAwME\nBAQFBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUW\nFhYXFxcYGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJyco\nKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6\nOjo7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tM\nTExNTU1OTk5PT09QUFBRUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1e\nXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29w\ncHBxcXFycnJzc3N0dHR1dXV2dnZ3d3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGC\ngoKDg4OEhISFhYWGhoaHh4eIiIiJiYmKioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OU\nlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJydnZ2enp6fn5+hoaGioqKjo6OkpKSlpaWmpqan\np6eoqKipqamqqqqrq6usrKytra2urq6vr6+wsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5\nubm6urq7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fIyMjJycnKysrL\ny8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc3Nzd\n3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq6urr6+vs7Ozt7e3u7u7v\n7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9/f3+/v7///+lpPcT\nAAAACXBIWXMAABJ0AAASdAHeZh94AAAejElEQVR4nO2deXwURdrHH0BJQsBAAoIGAuEKCOEO\nIKjIEUBwUUEQgXAs74KCXK4Krifyvuqrruy+XsuCi+u6HqvLu4Kyrgceq6wgrieKByiKoMgZ\nBEKSqc9Wd8/09BzVU5mpGmbSv+8f09dTT1V3fzN9TbqIAaABOtkNALUTiAW0ALGAFiAW0ALE\nAlqAWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQCWoBYQAsQC2gBYgEtQCygBYgFtACxgBYg\nFtACxAJagFhACxALaAFiAS1ALKAFiAW0ALGAFiAW0ALEAlqAWEALEAtoAWIBLUAsoAWIBbQA\nsYAWIBbQAsQCWoBYQAsQC2gBYgEt1AKxVhPRG9ZoF6KBjD3OZ1SepMYcntOyXv3VmpKv5yu2\nN7h+gbr+2ju7Xi9NVcaNV8Sq5PMeTUJj5vF6aKWm5GFi+evadgofdNFUZdxALLX0JGq1eJOm\n5GFi+et6hE9O+42mKuOmNor1zZo1a6rDgpIlVhHR9dqS+8UKrJ+/roeIsnza6oyX2ijWSqIM\nY6r6qWFtMtoMWV3B2GVkspHP3XPTyIKGvaes8++LL2Z2yuq/bjNRPT6xjKg1e6pbB8YqVp5b\nUP/Mfvcc4HMXEPU7uqg486zl1cduble/cOo3zuqd+fzV3BBYNpNo0OF5Z2Z0WHLQnPb9Y0r3\nzNN/ttbnrMs9vcX70zpklTzlF8taP39dfrJVb9VEqcViVY3yb/TiQw6xXm7snzv+uFHixSbm\nxBVBsf5g7PBjJf6oogOmWB37WWGDzEHzH4O1h+SLItbZVsm2O/mkb74/dka1oy7X9BaPZ5kL\nLoZYycQQKzPbpI5TrOV8fveyIXzeZPbFRj6xdHM523EaH+k82PhcwGP2c68aXjrqVLLFym1s\n7Ozr+Iweo9rwzyWmWHx+K3MH1muXyT9vtCsPzffF5gJuzeZvA0tnGkVO7dqAf47jk2v5cNBV\n5/LPRxx1uaW3+MJoX7vmZoQtlr+uzZuX8NXfvCUZW7om1BKxggTFGm0YZU7lVNvnWLOI6j7J\n2D7+3XAKP+QsJjrtQ8Zez7DFosxrn1zLOltfO2VEQyyxbq323cMHhdvZ962JRtuVh+Uzzntu\nCDbNEKtkD/tpHB9uYZUdTU/ZzUQtfwrW5ZbeYjxRo1eY78EQsey6HkrB76taLVYPooLHDrLy\n9evXH7fF4jtjqlFoW12ipxgrJLrOmJwXFOsZPqxevXr1bj7gx57upli5VYzt4gvv4wt/wc+J\n7MrD8kUR6wM+3N+If1+yTywv2MF6RC/bdbmmN6msb5TmjIRYyUR0jnWleWw59783G6fKfrFO\nnEL0pBnK98oydowfJ58zptbaYmX4ryd3PjJ/qHHwscTqyefsJSt4rmPPh+WLIlaBOTKCaAr7\nm8P/3wXrcklv8bn5fce5D2IlE5FYh6edYu3Ezq/YYn3NB6+boYP5CQrbxic3G1NbbLEKzKU7\nSs2SDQJi9WbWnl/PQvd8WL4oYlmhM4jOZ791iPXLYF0u6S1e4ku+M0b+CrGSifB2A/v+wZH1\njb2YsSPqN9bNbB+fu9aY+pvjdgOnkh9GG898bPuNscQKyyf8xuIHsWnGjc2sTZstvrLrii2W\ncQh91xh5AGIlE4FYBzZu3FjNDj9t3HS4P+o51hOM5RLNMSZnhIr1bx69lQ/HxRIrPJ/gHGsf\nP8e6w/hapN32MnmxKupah1njcgRiJQ+BWNv57L/yeUf4BfwqU6z7mXUV9zjfiwP5VdxOxvjZ\necY65nu4bqhYz/PodxjbUC+mWGH5ol0V7mblY4jqfMCONuOHQH7C92WXoqLNNRCLjbGuCh/A\nVWFSEYjl68T3de+JpcYZ+KfM15Co1+/3sO38u4OKzuNTtIgX+ICfvVObpuZpPguKZZw6ndq/\nj7Gwm/ueD8sXRSw6pVMG/5zJJ1fx4blXjuRFBvpqItZ7Rpo2zQhiJRXROda2poEz5buZdRgx\n7ry/FLhTPqnCKHFvXXNiTKhYvsnm3NZTiHJ+ct/zYfkixOre2lw62LibXj3bH9t/b00OhYzd\nYRUbDrGSifDk/eDycwozm3SZ+rYx8e34Ztazwt03Dm+V3bvsef+zwtfGts4dGvKs0ODo/3TM\n6rnowIs89a9i7PnQfBFiDdo5MTd70J3+X1u8OK5TRkHpnytYzcRiL5Tm5I54Zj3ESje4Qbka\n0hpiaUibDnhcrOsmT77LGM4hKtWQHmJ5lSv45dqSNzYavzp4REP6eMV6fLKTBaqblQw8Ltah\n/sFb4RqIV6zF5KS16mYlA4+LxU48MbJ9Zl6vae9pyY5DIQBqgVhACxALaAFiAS1ALKAFiAW0\nALGAFiAW0ALEAlqAWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQCWoBYQAsQC2gBYgEtQCyg\nBYgFtACxgBYgFtACxAJagFhACxALaAFiAS1ALKAFiAW0ALGAFiAW0ALEAlqAWEALEAtoAWKl\nLNdsOtktSASIlbK0ePxktyARIFbKArGAFiAW0ALEAlqAWEALEAtoAWIBLUAsEMGzs6R4wS0H\nxAIRlBXJeNXuCrccEAtEUDZTJmoixAI1A2JBLC1ALIilBYgFsbQAsSCWFiAWxNICxIJYWoBY\nEEsLEAti1ZDD78gwaqpMLogFbOaQFJ1lckEsYDNzwn4J2nSUyQWxgM3MMpmothDrZDcg3YBY\nckCsGgKx5EhMrCNfH/Ipaki6ALHkiFss35YF7bL55U9Wu/nvqWxQqgOx5IhXrIoJRI37lI4t\n7ZNLVFaptE0pDcSSI16xbqH+b1g6Vb1dSrera1CqA7HkiFesNq2O2eOV3dqraUw6ALHkiFes\nU8c7JubWV9GU9ABiyRH/N9Zxe7yqZ1s1jUkHIJYc8Yq1NHiOtamUlqlrUKoDseSIV6wTl/Gr\nwpLh40b0zSO6/ITSNqU0EEuOBO5jzSvMJKLMwnlbvHSTFGLJkdCdd9+hr3DnPSoQC490agjE\nkgOPdGoIxJIDj3RqCMSSA490agjEkkPPI53jD6+weeDauBuXikAsOfQ80vmmb2+bTlSr7nJB\nLDn0P9J5kyrirCMlgVhy6H+kA7GEQKxI5B/pQCwhECsK0o90IJYQiCVA6pEOxBICsaJy+P0D\n/rHvdriEQSwhECsKn55HVGfsN+Z4P7csEEsIxIpkVw4NmNic8r8yJiBWOBArXrGm0x8Zq15I\n51YziBUJxIpXrPbnGJ/Vl9LDDGJFArHiFavBNHOwu9HpByBWJBArXrGKu1aZw/tpTDXEigBi\nxSvWYpqxxxj6LqBFRyBWOBArXrGOFBO12cZH9vanJjkQKwyIFfd9rIrlg8/4tzFy9KYzCGKF\nAbFUvHitavsrLkshlhCIlQgQSwjESgSIJQRiJQLEEgKxEgFiCYFYiQCxhECsRIBYQiBWIkAs\nIRArESCWEIiVCBBLCMRKBIglBGIlAsQSArESAWIJgViJALGEQKxEgFhCIFYiQCwhECsRIJYQ\niJUIEEsIxEqEtBGrSqZ7+v1TpsjkglgQy+ZakqJIJhfE0t+BQNqIdcWFX0pQKNXTGcTS34FA\n+og1USaqI8SSQn8HAhBLCMSKRL4DAYglBGJFIt8nNMQSArEike8TGmIJ8YpYfzgkX64WdiAA\nsVTiFIsyJz4n2z1JLexAAGKpxCnWAwOJTl/wjlTXlrWwA4Hki1XScpgL9Yv9I3+RyZVqhJ5j\n7bi9K1Hn27+WKFj7OhBIvlgFTRe7kD3GGhbNk8mVakScvL+/uIDo/IcPyhSuXR0InASxznJb\nGjgUjqsdYh1bO62e8UX0y+PRwiM4vGlbjO5VIZYQ74h15OnLGxGdfe8nK7rQTNdyK24wPj8b\nzh3MWOh6NQmxhHhFrMfGZhENWG72NnGiUyPXcgONgrtzqWPZjGIqdvt2g1hCvCIW0cDffBuY\nGNvftZwp1nRaWsXPtO6iW1wiIZYQr4j122+FYRGYYrXvYp66+7qWhC09eu+dNldCLBFeEYux\nb1/kH6vcfwZjYYrVYJI1MblB2NJdA4N9QheR3GXASQdiqcQp1olF1J0P8unKqpjlTLF6+A+X\ngwtcInEoFOIVse6jvmv4YMNI+l3McgPp6odeur3OM8b4OnLbJRBLiFfE6tra+ilMZcdeMctN\nzDB/AM4vHcvH1W34iUskxBLiFbEaTPWPTGsYu2D1zg2rrp8wgLHddO7HboEQS4hXxCoa7B85\nv0MNMhz/xn05xBLiFbFmWqdM7P/rTFVYA8QS4hWxfsin0mUr7xhNzb5TWAPEEuIVsdj2SeYZ\n+eitKmuAWEI8IxZje996YkMN7r/LALGEeEgsaXJCcYmEWEI8I9bTE4f6iVluZR+iNt1tXCIh\nlhCviLWKKDvPInbByhG0RqoGiCXEK2Kdlf2q1D9SWKyFWGIgllOsDNf/RgpnV/azUnEQS4hX\nxGo5R0cNEEuIV8S6JX+vhhoglhCviFU5vehPn+0/YKCwBoglxCti5eTY70NUWAPEEuIVsWYH\nUVgDxBLiFbH0ALGEeEisio/f+qEG97JkgFhCPCPWd9MyidasK/1QZQ0QS4hXxPq+PXWdRGs2\nZjT5QmENEEuIV8SaT7dV76A1bEu9nyusAWIJ8YpYrXv6mCEWKylUWAPEEuIVsRqUMUusqeH/\n2ZwIEEuIV8Qq6VxliuXr01thDRBLiFfEWkpXHTPE+j0tUVgDxBLiFbFOnE0tRtKwvtT1qMIa\nIJYQr4jFjt2dT0R5NxxWWQPEEuIZsTiHP9qnuAaIJcRLYqkHYgnxiliTgyisAWIJ8YpY9q+x\nCtoprAFiCfGKWJUGJ3at6Tb0J4U1QCwhXhErwP5WV0uWrl19QkMshUQ9eb/ijNgF0Se0GxAr\nqlgzMmOWQ5/QrkCsKGJVvZDZLWY59AntCsRyipVtcSrRn2KWQ5/QrkAsp1ij/UxbF7sc+oR2\nBWLF34s9+oR2AWLFKxb6hHYFYjnFah6Kazn0Ce0KxHKKNfdsoua98okKBxq4F0Sf0G5ALKdY\nW5sMet8YDG0l09s4Q5/QYiCWU6xLz7B+4Vd+5vjowRHgkY4AiOUUq8UE/8iE/NgF8UjHDYjl\nFKvV2f6R/i1ilsMjHVcgVsihkJ4wh0/Q6Jjl8EjHFYjlFOvzHLpkxbMrLqH678Ys5/5Ip/zm\nxTZTIJYIr4jFNp1j/oC00/rY5dwf6ewZNcymBH1Ci/CMWMz33pP3PPKm2xlTADzScQVixfvi\nNTzScQVixfviNTzScQVixf3iNTzScQNiJfTiNTzSEQGx8OI1G4ilErx4zQZiqQQvXrOBWCrB\ni9dsIJZK4n3xGvqEdgVixfviNfQJ7QrEivvFa+gT2g2I5RTrvsdqUhJ9QrsAsUL+E1qi8/og\n6BPaBYgVeuf9nxpqgFhCvCJW9a0tV376I7rudQViyeEUKy+vLrrujQXEkgNd99qkqljFp/eW\n4VGZGpNHQKy5K3XVALGEyImV33KFBN1lX++ZJAJi0Tjj875p6muAWEIkxXK7/WwzJqXFGqeh\nPwGIJQRiJQLEEgKxEgFiCYFYiQCxhECsRIBYQiBWIkAsIZ4QK/8yTj5dZqGwBoglxBNihaKw\nhhQQ6xOZO4wrzhshkwtiyRFQaHMoCmtIAbHmN5F5JpLVWiYXxJLDEz2szhsnE3VWgUwUxJID\nYtlALJVALBuIpRKIZQOxVAKxbCCWSiCWDcRSCcSygVgqgVg2EEslEMsGYqkEYtlALJVALBuI\npZLExEqT3r8gVvKJW6x06v0LYiWfeMVKq96/IFbyiVestOr9C2Iln3jFcu/9ywnEEgKxInHv\n/csJxBICsSJJq96/IFbyiVestOr9C2Iln3jFSqvevyBW8kngPlb69P4FsZJPQnfehb1/HVww\ny+YiiCUCYok5vGlblLujeyeNtxly8vuEhljJJ16xVtxgfH42nB8MMxYecovEoVAIxIpkoFFw\ndy51LJtRTMVu30kQSwjEisQUazotreJnWnfRLS6REEsIxIrEFKt9F/PU3de1xCUSYgmBWJGY\nYjWYZE1MduuRFWIJgViRmGL16G9NDC5wiYRYQiBWJAPp6odeur3OM8b4OnJ7sxTEEgKxIpmY\nYb5HqxFj5ePqNvzEJRJiCYFYUajeuWHV9RMGMLabzv3YLRBiCYFYbhz/xn05xBICsRIBYgmB\nWIkAsYRArESAWEIgViJALCEQKxEglhCIlQgQSwjESgSIJQRiJQLEEgKxEgFiCVEpVufmUl2S\n/1kmlwoglk16i9WiQKa/oK6LZXKpAGLZpLlYvWWiLoBYKoFYASCWUiBWAIglycsTxkvQvp9M\nLoilkjQX6+rWsyTIzZfJBbFUku5ijZGJ6g6x/EAsSSCWHQ2xVAKx7GiIpRKIZUdDLJVALDsa\nYqkEYtnREEslEMuOhlgqgVh2NMRSCcSyoyGWSiCWHQ2xVAKx7GiIpRKIZUdDLJVALDsaYqkE\nYtnREEslEMuOhlgqgVh2NMRSCcSyoyGWSiCWHQ2x5Hi4rQw5HWRyQawAhY2lNusjMrncSVmx\nFvd4SoK2LWRyQawAzTrIbNXiG2VyuZO6Yl0gE9UbYgWi5cTqLxM17KSLdeTr6P0VOoFYQiBW\nFHxbFrTLJqKsdvPfcw2EWEIgViQVE4ga9ykdW9onl6gsSl+YNhBLCMSK5JZgL/Zvl9LtLpER\nYr0q88/L4zsVyzQDYtnRtUSsNq2O2eOV3dqHLd0/x6VP6BtJikyZ/3FumiX1n9DZMlFNGspE\n5TSSiWqUIxPVsInb0gZDrWF2rkyurKYyUZmny0Tln0SxTh3vmJhbP2ypU6xpA8IWvimzbrPG\nDJWJunCYTNToUpmoUcNloi4YIRM1YqRM1PBRbksDYpWOlsk17EKZqKFjZKJmvRmnFQ7i/8YK\ndtdb1VPq8ABqRuBQmJ7EK9bS4DnWplJapq5BIIA3xTpxGb8qLBk+bkTfPKLLTyhtEzDxpljM\nt2VeYaZxjl04b0vMm6QgDjwqloHv0Fex77yDOPGwWEAnEAtoAWIBLUAsLVwsd38e6GBC4vsv\nZcVaNOAdnYwv1Zp+1EWJ51j3tnDR+ZMST+9CXwU/YE5ZseR+3RA3cxT8UbpQNlNr+osXak1/\n8n/opxGI5QLEih+I5QLEih+I5QLEih+I5QLEih+I5QLEih+I5QLEih+I5QLEih+I5QLEih+I\n5QLEip+bLtKafuEUrelnXqk1/YTrtKYftTTxHCkrVvn3WtMf/FFr+n0HtKbfe0hr+j1HEs+R\nsmKB9AZiAS1ALKAFiAW0ALGAFiAW0ALEAlqAWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQC\nWkhJsVbmOKdOLGtbv+1tqt5GGZFt4kCTFRpSp03L/ajc7qkoVmWJcwV9l1PLS/NpoppXB0Zk\nq86wXrByg/rUadNyP0q3e+qJ9d1zI8m5gluo3zF2rC+9qyR7RLaddLWSxFFSp03LTRRv99QT\ny+j5ybmC8+gN/vkGqfn/gYhsG+ghJYmjpE6blpso3u6pJ9aza9a0ca5g28bG++QrG4d3qxIf\nEdlW0UtKEkdJnTYtN1G83VNPLE53xwr6MvuYwz7ZKjJHZrue7ujVoOPPd6tPnTYtt1G53VNe\nrEM03ByWkoJ/HYmSbQLV6Xt5Z8r9XHnqtGm5jcrtnvJifUXjzOFY+lpB5shsAxo9zS+wbqUR\nylOnTcttVG73lBGr6nPOd9Z46F+OteFKKaH/pfOnF2Wr6kjliaRnURqqqOWi9H4UtNxG5XZP\nGbH2Grdk/P/9HHqs72sO+zRI6HaQP70wWxltSiQ9i9JQRS0XpQ+QeMttVG73lBHr+F84/n7y\nnCvICvOq+WdVXjsl6cOzHd9t/b3PoG0J5WdRGqqm5aJsClseQOV2TxmxnISs4FXmX+TbNF9J\n6vBsO61TCV9xRpXq1OnT8gAqt3sqi3V0xy5m3AEeXsUqh9O/laR2ZLPSn1P3Ob537qIF6lOn\nTcsDqNzuqSzWi2R0s+27jHpd1YMmq0ntyGal/yibhkwupuLET7AjUqdNywOo3O6pLxarWNom\na+Cdqn4jEMzmT791Qqus3jcdi1EsrtRp03I/Krd7SooF0h+IBbQAsYAWIBbQAsQCWoBYQAsQ\nC2gBYgEtQCygBYgFtACxgBYgFtACxAJagFhACxALaAFiAS1ALKAFiAW0ALGAFiAW0ALEAlqA\nWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQCWoBYQAsQC2gBYgEtQCyghfQRa/OlRZmth/65\nOnZkb1ovkU8uys/A5v4Cq1plfc7+XpTxvHxZAbOurmEbZDDaaXPodDVvP42LtBHrXqo7qGzE\nacYbV2MRa3etpUclokIIiLW7XrP5P/iaZs36WL5sdN5o9INusdidJcpeqFxj0kWsD+udsZUP\nDlxM98SM3bXjqOtyS6xYUSEYO8wo8E+6nbEfaap8SQG+kkU1bYMMoWKVN3xUbfoakC5i/YYe\nNIf76wxLOJclVo0I7LA36D6jl4u5Cbdho6KXdIcRKhabVqKmd9c4SBexrqbHrZH7DcFOLOuX\nXbiIH0tY9eq+Obnn/T1kbDYd4AZe2TW75zU/GZM5lbcWZHZ9OJBqhNH7yV4zam7O8YVFTS/e\n89OV7RsO/pCFZPbz8cVn5k9439hhvMBkCrDGGTo7r3phw/tC5jjq3PuLTtnd7j8Rmnxqkc/f\n0vDmOVbEWSCYJfqaBdvpKP8P2qx6R8iSLmL9kTo8b5+3Hx9Anab0oA67GbuNci4a36Dua84x\nY3ftKqA+ZcXU6aCx+Wfkz5mdTc/4S/9jAf1i9TFLrOwLel57HnUvOeuaUupQFZLZ4tUGdPb4\nM04rsMR67XqavHr1/9GQ1V87Q2fn3UpNHguZE6xzR8s655e1pkUhyaubzTayW2KFNs+xIo4C\nwSzR18zRzmB5dqTubcnaQeGki1iV/Hum/YJnD5oT99DcKuZbStOZL691OWOvhYyZu+u/6Nf8\nROY6usmYLOJ/86/SxEAu61BoikWjK/npDp1zjPmG0XZnZovq7vQkv7waRJZYzkOhI3R23fzX\nw+YE65xMTzN2rIS+dS5/n1YzfxvCm+dYEUeBYJaoa+Zop6M8Yz3OT9YOCiddxGJVT1+aR1Tv\nfOMPO7+F8db86i5ZJyrqtq3kYxs/ZsExY3dV1OtqfL8da9HMmPwTH/VlDw2kcor1Fh/7JT3H\nP5cZh41gZiv0X3SJMXg/iliO0Nn0exY+J1DnXuukcE33F53L/2hW7BcrtHmOFQkWCGaJvmaO\ndjrKMzaxsY59IUPaiMWpfm95N6LF7DCN3GEwiT5mP6Muyz8yj5HBMb67ttFVZpFL6CCf/MwY\nzYsq1vd87Aaz/6y7uVjOzCaP0gpz2CJCLGfobPqUhc8J1PkWLfNX61x+l9VllyVWWPPsFXEU\nCGaJvmbOdgY3BOMrqLDjihqRJmL5yq3rct8LTeps/cg+hX6LlS9pQdRiwY/MMcZ31wb/jphD\nH/HJ/cZodLH2MkMso/dbQyxnZpO76Vlz2CtCLGfobOK1hs0J1PkErfRX61y+hMw+Py2xwppn\nr4ijQDBL9DVztjO4IRj7Fe1Stg9qRpqIVd2om3/sV/TIPhq2xsK8Lnzn172oZ7VjzPGNNY72\nWReJcmKFZuY8YR3k2JkRYjlDrRoi5xh1bqA7/NU6l4d8Y4U1z14RR4Fgluhr5mync5PgGysW\n55zymTVyBb3EcvuZo/9a5/vylpf5iG8IbQ+OWedYxcbl/PEzc1lNxHJktkLfpbHGYGuUcyxH\nqL+GyDm8zm/pQmNsQ/4q5/KQc6zQ5jlWJFggmCX6mjna6SiPc6zYrKIub/NB9TP1mx7mJqzi\n41syhrEd1KeCb+Xe9cqDY/6rwuU8+hpaElUso3hUsYKZLXx9jaut8qFRxHKE+muInGPUOYr+\nxi89RtJ7zuUhV4WhzXOsiKNAMEvUNXO001EeV4Wx8c0kKhxyXj5lvcrPartQ32l96zX+gPlG\nU8ef/yyX5jvGrPtYrahvWVfrbk+4WK9Q8fXl0cUKZvbzz4Z09oT8/BGRYjlC/TVEzjHq3Nq0\nzuDpRXRFyHLnfayw5jlWxFEgmCX6mgXb6SjPjtTDfayYbLioKLN5v8V7jPGj1/XIajPd0OHg\n9R2zcvutrHKO+e+8d2nQ/ZojLIpYFWMz8/ZFF8uR2c8nl+S3GP/l3EixHKH+GiLnmHXumtIu\nu8eDVaHJHXfew5sXXBFngWCW6GsWbKejPO68e46N9G4SapneB88KPYavZKH+So40wq8bPMfr\njb7XXsf/9sHvsbzHrEW6azh0ejIOtwIgFtACxAJagFhACxALaAFiAS1ALKAFiAW0ALGAFiAW\n0ALEAlqAWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQCWoBYQAsQC2gBYgEtQCygBYgFtACx\ngBYgFtACxAJagFhAC/8Bi2DPPEdtD8oAAAAASUVORK5CYII=",
      "text/plain": [
       "Plot with title “Histogram of perm_diff”"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "perm_diff <- rep(0, 1000)\n",
    "for(i in 1:1000)\n",
    "    perm_diff[i] = perm_func(session_times[, 'Time'], 21, 15)\n",
    "hist(perm_diff, xlab='Session time difference(in seconds)')\n",
    "abline(v = mean_b - mean_a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Figure3-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:02:17.780706Z",
     "start_time": "2019-04-01T16:02:06.249Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAHgCAMAAABOyeNrAAADAFBMVEUAAAABAQECAgIDAwME\nBAQFBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUW\nFhYXFxcYGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJyco\nKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6\nOjo7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tM\nTExNTU1OTk5PT09QUFBRUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1e\nXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29w\ncHBxcXFycnJzc3N0dHR1dXV2dnZ3d3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGC\ngoKDg4OEhISFhYWGhoaHh4eIiIiJiYmKioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OU\nlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJydnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWm\npqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+wsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4\nuLi5ubm6urq7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fIyMjJycnK\nysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc\n3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq6urr6+vs7Ozt7e3u\n7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9/f3+/v7////i\nsF19AAAACXBIWXMAABJ0AAASdAHeZh94AAAegUlEQVR4nO2deWBU1dmHXwhkBQKJyhICSUBA\n2SHsKiIEEBXLEkBZIi6gIJttAbcq8lWt1WIrtlVQaqt1+VSq0FJFxeUTKhAV64YiKLKoKDuE\nkGTOd+6dOZOZuTNn7sycM5nk/p4/5i7nvOe9y5OZM/dO7iEGgAaopjcA1E0gFtACxAJagFhA\nCxALaAFiAS1ALKAFiAW0ALGAFiAW0ALEAlqAWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQC\nWoBYQAsQC2gBYgEtQCygBYgFtACxgBYgFtACxAJagFhACxALaAFiAS1ALKAFiAW0ALGAFiAW\n0ALEAlqAWEALEAtoAWIBLUAsoAWIBbRQB8RaRUTvuGc7Ew1i7Gm+oqKGNuborNZJyas0Nb6O\n79iB6v0TuV7snZHUS1PKqHGKWBV83d/isDFzeB5aoanxALE8ubY34JPOmlJGDcRSS0+i3EWb\nNTUeIJYn1xN8seRBTSmjpi6K9e3q1aurAirFS6yORDdra9wjltg/T64/EaW5tOWMlroo1gqi\nFGOp6rlheSl5F60qZ2wimWzia7+7fWSbRr2nrPWcix3XdErrv3YLURJfWErUlj3X7WzGylec\n3ya5Vb/7D/G184j6nVzQNfXcZVVlv2qXnD/tW9/0vu150twqyq4hGnx0TquUsxcfNpddr07p\nnnrWZWtcvrnkzbvZVnJ2Wp/nPGK598+Ty0OG6qMaK3VYrMpRnoPe9YiPWK839awtPmVErG9m\nLlxfLdbjxgkv6+Op1fGQKVaHfu5qg81J8x+rs/u1F0SsAe7Igt180TXXU3d6lU8uafNunk4z\nC34GseKJIVZqhkk9X7GW8fXdp17E101mOzbxhSVbjrFdTfjMOUOM13m8zkHuVaPxoxqSV6ys\npsbJXshX9BiVx18Xm2Lx9bnmCUxql8pfb/Mm929vx5Y23Jote0TpNUZIwy7p/HUcX1zDp4Nv\nPJ+/PuGTS9a8mx3G9rVrbtbwiuXJtWXLYr77W0rjcaQjoY6IVU21WJcYRplLmVXePtYMovrP\nMvYTf29owD9yFhE1+S9jb6d4xaLUXz67hp3jftuZSnSRW6w7q1z380n+TvZ9W6JLvMkD2jP6\nPbdWb5ohVp/v2IlxfFrKKjqYnrJfEbU+UZ1L1rybYqLGbzDXH/3E8ub6UwK+X9VpsXoQtXnq\nMDu2bt26U16x+MmYZgRtr0/0HGP5RAuNxTnVYr3Ap1WrVq3azyf8s6e7KVZWJWN7eeFDvPA6\n3ifyJg9oL4hYH/Hpwcb8/ZJ95vaCHU4iet2bS9q8SUWyEc0ZCbHiSag+1g3mZ8v5/7PF6Cp7\nxDrdgOhZsyo/K0tZGf+c/KextMYrVorn++TuJ+YONT583GL15GsOkLvybJ8zH9BeELHamDMj\niKawl3z8/3N1Lknzbr403+84D0GseBJKrKMlDdwn8Zw3vGJ9wydvm1WH8A4K284XtxhLpV6x\n3CLsKjIj04VYvZn7zK9j/mc+oL0gYrmrTie6kP3eR6yfV+eSNO/mNV6yz5h5EWLFk5CXG9j3\nfxyZbJzFlF1B37F+xX7ia9cYSy/5XG7gVPCP0abXPLXztnBiBbQX8h2Lf4iVGBc20zZvcfO1\nN1d4sYyP0PeNmYchVjwJIdahTZs2VbGjzxsXHZYH7WM9w1gW0Sxjcbq/WB/w2p/y6bhwYgW2\nF6KP9RPvY91jvC3Sfm+ZfbHK67s/Zo2vIxArfoQQaydf/SJfd5x/gV9pirWcub/FPc3P4iD+\nLW43Y7x3nrKWuR6r7y/Wv3jtrYxtSAorVkB7wb4V7mfHRhPV+4idPJN/BPIO31edO3bcEoFY\nbLT7W+HD+FYYV0KI5erEz3XvSUVGD/xz5mpE1OvR79hO/t5BHS/gS7SAB3zEe++Ud4bZzWfV\nYhldp4b9C43CbvIzH9BeELGoQacU/noNX1zJp+ffMJKHDHJFItaHRjN5ZxLEiiuh+ljbzxA9\n5d8y98eIceX9NXGl/MpyI+J39c2F0f5iuSaba9tOIco8IT/zAe1ZxOre1iwdYlxNr5rpqdv/\nQCQfhYzd4w4bDrHiScjO++Fl5+WnNus87T1jYU/xme57hftvG56b0Xvqvzz3Ct8a2zZrqN+9\nQoOTv+6Q1nPBofW86VvCnHn/9ixiDd49KStj8L2eX1usH9cppU3R38tZZGKxV4oys0a8sA5i\n1Ta4QVkamjXE0tBsbcDhYi2cPPk+YzqLqEhD8xDLqVzPv64tfmeT8auDJzQ0H61YT0/2ZZ7i\nrYoLDhfrSP/qS+EaiFasReRLW7UbFR8cLhY7/czI9qnZvUo+1NI6PgoBUAvEAlqAWEALEAto\nAWIBLUAsoAWIBbQAsYAWIBbQAsQCWoBYQAsQC2gBYgEtQCygBYgFtACxgBYgFtACxAJagFhA\nCxALaAFiAS1ALKAFiAW0ALGAFiAW0ALEAlqAWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQC\nWoBYQAsQK34seL+mtyCO6BHryK2LvMwPHIjWsWS9UNNbEEf0iPXDFcVeLqJTWnLUPiCWUt6l\ncu05agcQSykQSwCxlAKxBBBLKRBLALGUArEEEEspEEsAsZTiPLE2zAhOSpFn5uGa3sI4ALHU\nc1NbuViDOtf0FsYBiKWem0YHXy8+CpdDLBVALAHEUgrEEkAspUAsAcRSCsQSQCylQCwBxFIK\nxBJALKVALAHEUgrEEkAspUAsAcRSCsQSQCylQCwBxFIKxBJALKVALAHEUgrEEkAspUAsAcRS\nCsQSQCylQCwBxFIKxBJALKVALAHEUgrEEkAspUAsAcRSCsQSQCylQCwBxLLN8W+OuMLVgVgC\niGUHV+m8dhlElNZu7ofSihBLALFsUD6BqGlh0diiwiyiqRWSmhBLALFscAf1f8etU+V7RXS3\npCbEEkAsG+TllnnnK7q1l9SEWAKIZYOGxT4Ls5MlNSGWAGLZIC+3+iHblT0LJDUhlgBi2WBJ\ndR9rcxEtldSEWAKIZYPTE/m3wj7Dx43om010xWlJTYglgFh2cJXOyU8lotT8OaXSi6QQSwCx\n7OI68jWuvFuBWAy3dHQAsRhu6egAYjHc0omG+c3kpLYLHgexbODkWzqjL31OSkGL4HEQywZO\nvqUz+iZ5eW+IpeuWzldp5ENdGwgTYtlAzy0d15vrvTyIdywPEMsGTr6lA7FsgFs6kQOxbIBb\nOpEDsWyAWzqRA7FsEINYR7cd8szt2yWpBrEEEMsOn19AVG/st+Z8P1krEEsAsWywN5MGTmpO\nOV8bCxDLD4jFohfrKvorY1Xz6fwqBrECgFgserHan2e8Vo2nxxjECgBisejFSi8xJ/sbn3UI\nYgUAsVj0YnXtUmlOl9PoKojlD8Ri0Yu1iKZ/Z0xdF9OC4xDLD4jFohfreFeivO185kB/apYJ\nsXyBWCyG61jly4a0/MCYOXl7S4JYvkAspubBa5U735CUQiwBxFIKxBJALKVALAHEUgrEEkAs\npUAsAcRSCsQSQCylQCwBxFIKxBJALKVALAHEUgrEEkAspUAsAcRSCsQSQCylQCwBxFIKxBJA\nLKVALIEQ68GCrXJ2qN3eGgFiRU6sYl1BYWiidntrhACxyj/Z+EPYf5qPDIglEGJNSJHHr0uN\nfJsSDj+x9pWkEq1eW/RflRkglsCxYn3fnrpcSas3pTRT+SEPsQSOFWsu3VW1i1az0qSrFWaA\nWALHitW2p4sZYrE++QozQCyBY8VKn8rcYk1LV5gBYgkcK1afcypNsVyFvRVmgFgCx4q1hG4s\nM8R6lBYrzACxBI4V6/QAajGShvWlLicVZqh9Ym2TDzzxXOH18niIxQKuY5X9NoeIsm89qjJD\n7ROrW7p8rJz6hfJ4iMWst3SOfvyT4gy1T6zOy+XlLcJ0QSEWCxRrz3r+slI+TFykQCyBY8U6\nvYC680kO3VCpMAPEEjhWrIeo72o+2TCS/qwwA8QSOFasLm3dQ8VVdOilMAPEEjhWrPRpnpmS\nRgozQCyBY8XqOMQzc+HZCjNALIFjxbqmnnvP/1FvWvDKtin/yyNeFkIsD44V64ccKlq64p5L\n6Mx9Mba6u1OBl1a1boRViKUAv+tYO680f3J9yacqM+CjUOBcsRg7sPGZDXvUZoBYAieLpQGI\nJXCuWM9PGupBYQaIJXCsWCuJMrLdKMwAsQSOFevcjDcV/0+hAcQSOFaslDA/YIsOiCVwrFit\nZ+nIALEEjhXrjpwDGjJALIFjxaq4quOTXxw8ZKAwA8QSOFaszEzv804UZoBYAseKNbMahRkg\nlsCxYukBYgmcLBaej8UglhLwfCwrEEsBeD6WFYilADwfywrEUgCej2UFYikAz8eyArEUgOdj\nWYFYCsDzsaxALAXg+VhWIJYC8HwsKxBLAXg+lhWIpQDcK7QCsRTgK9bkahRmgFgCx4rl/TVW\nm3YKM0AsgWPFqjA4vXd1t6EnFGaAWALHiiU4mBvmQeYRAbEEjheLXd9SYQaIJYBY01XuGcQS\nOF2syldSuynMALEEjhUrw01DoidtRh//5kjYHzJDLIFjxbrEQ8laG4Gu0nntMogord1c+YAD\nEEvgWLEioXwCUdPCorFFhVlEUyskNSGWAGLZ4A7q/45bp8r3iuhuSU2IJXCsWM39kcbl5ZZ5\n5yu6tZfUhFgCx4o1ewBR8145RPmDDKRxDYt9A5MlNSGWwLFifdps8DZjMjT3m7BxebnVD9mu\n7FkgqQmxBI4Va3xL9y/8jrUqDl7ZhyXVfazNRbRUUhNiCRwrVosJnpkJOWHjTk/k3wr7DB83\nom820RWnJTUhlsCxYuUO8Mz0D3FkfHGVzslPJaLU/Dml0oukEEvgWLHG0zPm9Bm6xF6w68jX\nuPJuBWIxf7G+zKQxj7z8yBhKft9mNG7pBANisYALpJvPM39A2mmdjUDc0gkJxGKBV95dHz57\n/xPvym7QCHBLJzQQi0X/4DXc0gkNxGLRP3gNt3RCA7FY9A9ek9/S2Tugt5eOGAjTg2PFiuTB\na/JbOmXL7vVyA96xPDhWrEgevIZbOqGBWCz6B6/hlk5oIBaL4cFruKUTEojFYnzwGm7pBAVi\nMTx4LRgQSwF48JoViKUAPHjNCsRSgK9YDz2lIwPEEjhWrIwIBq/P9EdSE2IJHCvWXPo/23Er\nConyunuR1IRYAseKVXVn6xWf/2hz6N6KEcalVBtALIFjxcrOrh/B0L1rIFYIIBaLZejevRkv\n28oAsQSOFGv2Cl0ZIJbAkWLROOP1oRL1GSCWwMFijdMwngDEEkAspSSeWN2byUkK08WEWDZw\noliptzwnpeEEeTzEsoEjxQrzf5MpECt2IJaVmhZrOYWh2WF5A4kAxLJS02ItpfVS/kJ75A0k\nAl6xciZycmiiG4UZIJbAvljy8i9qlVj+KMwAsQSOFGuLPwozQCyBI8XSB8QSQCylQCwBxFIK\nxBJALKVALAHEUgrEEkAspUAsAcRSCsQSQCylQCwBxFIKxBJALKVALAHEUgrEEkAspUAsAcRS\nCsQSQCzb1M6xdCBWHIharFo8lg7EigPRilWbx9KBWHEgWrFq81g6ECsORCtWbR5LB2LFgWjF\nko+l4wvEEkAsG8jH0vEFYgkglg1q81g6ECsORCtWbR5LB2LFgRiuY9XasXQgVhyI6cp76LF0\ntm318jjE8gCxIuDo5u1Bro7uSPL9t+pEG2EVYsWBaMV65Fbj9Yvh3JuU+UcsxccPevk33rE8\nQCwbDDIC92dRh6nTu1JX2XsS+lgCiGUDU6yraEkl72ndR3dIakIsAcSygSlW+85m193VpY+k\nJsQSQCwbmGKlX+lemCwbQxpiCSCWDUyxevR3LwxpI6kJsQQQywaD6KY/vXZ3PfNQraVJkpoQ\nSwCxbDApxbxE1ZixY+PqN/pMUhNiCSCWHap2b1h584SBjO2n8z+RVYRYAogVEae+lZdDLAHE\nUgrEEkAspUAsAcRSCsQSQCylQCwBxFIKxBJALKVALAHEUgrEEkAspUAsAcRSCsQSQCylQCwB\nxFIKxBJALKXEXSzX3x+R0/Af8gYglgLqoFj7KLdACv1a3gDEUkAdFGsPfSGvIH3UBINYSoBY\nViCWAiCWFYilAIhlBWIpAGJZgVgKgFhWIJYCIJYViKUAiGUFYikAYlmBWAqAWFYSXazP6JZ7\npfz2mLyBeACxrCS6WOup3zAZQ+lteQPxAGJZSXSxXqWt0vIqekveQDyAWFYglgIglpXaL9aV\ni6TcvFueQAUQy0ptF6uCOkr7YMPSVskTqABiWan9Yv1B3kD+4/JyFUAsKxBLARDLCsRSQC0U\n69At8q7pLPpY3gDEgljBeIvGF8sYRq/KG4BYECsYb1GVtHwrxKr1Yh3/JsToXz5ALAHEsoOr\ndF67DCJKazf3Q2lFiCWAWDYon0DUtLBobFFhFtHUICPLeYlYrC9nzpByGcSqw2LdUT0m9HtF\ndLekpkWsjXJvZgxOlvbNi3uQzGOIFV6s7MFhTsFGebwdoh/Fvsw7X9GtfUDpwVnVG3l5oFi3\nURjqy/d6IF0rLR9Do+QNUKG8PKlAXp52hrz8jPTg61OK3NOCJHl8IcnLR9EYafm1NFDeQP1w\nZ+C2KK3wIVqxGhb7LMxODij1FatkYEDhu/K9njFxgLx8Wi95+bU9rpZXKJwiLz+vWF4+7FJ5\n+aXDgq8XYhWfJ4+fEkb8q3vI/7Jm9JomLx8wUV4+490orfAh+nes6sEvK3sWxL4hDkB8FDqC\naMVaUt3H2lwUrtMCTCCWDU5P5N8K+wwfN6JvNtEVp5VuU10FYtnBVTonP5X381Lz55SGvUgK\nDCCWXVxHvg5/5R0IIBbQAsQCWoBYQAsQq2a5K9x1YaCbMPe07JCAYq1otTUm/kUvxtZAwcLY\n4idfEHz9ms324v+QHFv+zfRobA30XRT7WUxAsR7Pjy0+7G/ew9F5eWzxN42OLX5damzxMf/D\n6rAavFeoEYgFsbQAsSCWFiAWxNICxIJYWoBYEEsLEAtiaQFiQSwtQCyIpQWIBbG08GTH2OJ/\noK9ja6Dno7HFLx4fW/zrTWKLdzXcFFsDo5bEFm+QgGKdjvVBhl/FGP/tqfB1ZBz9Ibb4qp2x\nxbOdMf768rvjMW4AS0ixQF0AYgEtQCygBYgFtACxgBYgFtACxAJagFhACxALaAFiAS1ALKAF\niAW0ALGAFiAW0ALEAlpIJLFOLy1ILrjL57GTe6a2T+9687Go408s6pZ+9vR9UcdzVmRGGxys\nNV3JgzYQ695HePQDSCCxXFdQ6/E5NMn7K7V9zejCknOot3y8gNDx5V2p87SBlLk9ynhORR+b\n59YSHKw1XcmDNhDr3kd49ANJILFKqV8ZK+tL74sVM+gxxion0soo45dRSSVjT9DgKOPZvn+O\nJJvn1hJsbU1f8qANxLr3ER79QBJIrDn0Dn99h+aLFQU5xqA579HMKOOH0H5jMrDe0ejimTEI\nlc1zawm2tqYvedAGYt37CI9+IAkkVkFT4123oqkYP6Xi3CnGZDtNjC6etcwzJ5NoW3Tx7OXV\nq/NsnltLsLU1fcmDNhDj3kd69ANJHLFcqYXmtDDDf/1vwg05FDL+A7N7UdW83qHo83e3d24t\nwaH2Rkfy4A2o2Hv7R99C4oh1hIab0yLy+R+R1TP70xh7/zQTNJ4f2fk0NoZ4m+fWEhxia7Qk\nl+SLbe8jOPoWEkesr2mcOR1L31SvnE2Udl9l9PFsfzHlfBtDvM1zawkO3pqe5KEbiHHvIzj6\nFhJArMovOfv438wIc7GIjvgUntr2M1oQdbzr4SZ03q5Y8tt+xwoIDt6a/fhIkodqwNbeSzfA\nxtEPRQKIdcB4Tu/l/FO+r7lYmO5/6aesZYr8KmPo+B9H0Vkrw/7FSfPb7mMFBIfaG7vxkSQP\n0YC9vZdugI2jH4oEEOvU/3LeZSw/2/h+W5ndzrP+/clrzOlQ+i6qeHayP11qo+caMt7A7rm1\nBAdtLYL4SJIHbcDm3oeKt3v0Q5EAYglupM3MuHAy17O8g0qMiSs/097V68B4djvNl48eHSbe\nwO65tQQHbU1T8qANxLj3kR79QBJIrFIaXskqhtMH/K9t116+TwXJW/nrg3avpATGV7ZqFtEz\nCALjTeyeW0uwzwrtyYM1EOveR3r0A0kgsVwTqdeNPWgyn11P3fnrK/UaDJ/ak1rZfC8OjN9J\nmf3c2LsRa8lvYPfcWoJ9VmhPHqyBmPc+wqMfSAKJxcqX5KUNutfoK3qO7eaLW6d3/8XhKOPf\n8I7gsSvK/CyCc2sJrl6hP3mQBmLf+wiPfgCJJBaoQ0AsoAWIBbQAsYAWIBbQAsQCWoBYQAsQ\nC2gBYgEtQCygBYgFtACxgBYgFtACxAJagFhACxALaAFiAS1ALKAFiAW0ALGAFiAW0ALEAlqA\nWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQCWoBYQAsQC2gBYgEt1F2x3ryidUreyH/H2kxv\nWqdgY2bcpKARduQsm080TQDqqlhVc4lyhp1L0Q5e5SVysdbQ3wJXvdP4hxg3w93ovX2iHCci\n/tRVsW6nAuOve1sHeiK2hvbuOhlhhFUsV58oh3cIbPRYI4uziUodFeuLBmccMGc+oh7xzm0V\na5Pdh3KHbbSkT3RPXY8/dVSshXSvZ25S9+8ZO3hDl4yevzjBF2dmVtzZJrXLY4xNNUd+ZM/T\nQmM85H4Z+QuMj6uZ2VXzGz3Eqlb1zcy6wOigzaRDweM9eALYB+NbJ+eMKWVshPGk4gO+bbJp\nHQ0dmpdsn5bbepw52pslITtwXaeMbstP+5d5k4lGX6Ut8TqEMVJHxRpCvk8339uGCqd2pU6H\njXM1PWfWzAx6gb1MPzfKJtB/2amB1GlKDzp7v3Ge76RmT7G7KPPy4vT6b7nFChrvwRPwZWbS\nxdO6UOa37NV5dN2qMt82q840O3rNB2e1LO5Pjf/DgiTc1brehVPbGiMi+ZZ5k3kaZcfr3xXn\nIxktdVSsXL+xhq6lB3hHZyHdbnjSkb8XvEmT2Kkm7XidE+k9GbufZlcy1xK6ipfXz3mb181u\ne4yxt8wVXKyg8R7cAbxP9zx/fcDo0bk/tXza3EarjJrN6YIjjD1JF7isCdlkI76sD+3xK6tO\nJj5fe1wYt2MYG3VTrIp6HXyWypO6GKPKlLU40zhXT/JZV8ZQ47PwI+OT8HeM5bTgbwasqnPa\naV7+qBFRv6CCr9j0iSlWiHg37gD22gpj5NtXaJlwwKfNv9JGo0pz+tCYjKJPrQkP1BtmlK3u\nvt6/zJtMiDWpqcbDppK6KRZr1sxnYTvdaE7H0GF+rr4wZrPNc8U/ViYm7WdHaeQugyvpE17+\nuVF+GXVe9rE5xhEXK0S8G08A5+TmB7t4xfJt8z4y+1XNW5rV/kAvWRNupKWeVvzLfDbWLdZs\nKlN9rPRQR8XqRz955l6ZuW6D56zNoo/5uTpozBrn6lSTXuxExsWMfewdHWQjL//RKD+2uAVR\ni3k/mmKFiHfjCTh8U+ek+t0u9orl2+Zid4eveW8z4AVabk34DK3wtOdf5k0mxLqFxAhOCU4d\nFesa+rNnbiy9Jt5xxnHbzC95HjGm0TfP09OM/UTDVrv5QZTzz6KtD/SinlW+71iB8SaeFZfT\ndWuPs01esXzbFO9YuWbAw/S8NeEGusfTXpCNwTtW4vAutXSPLvRVWsrJ8qSuRlf+VKss5ifG\nGvr9xMbGNYSsfmbd/6x1ecq/uuN1/uq6iHa6+1jB403cK44lm0MqP13dx/JpU/Sx6u0wJpfz\nrp0l4R661FixIWeltcxPLPSxaphpdHYpn2zvTXcb3wr5+a76BS32F+NUZv+Mq42lW2klfy1N\nGSbKd1FhOS/vnXTM860waLyJe8VBOp+7t7ujkW2N2ZpPm95vhaNOGu71c1kT8i79S4xVjuT9\ne2uZW6yVZjp8K6xhysYQtRzatQGNrmRsby71ndrFfR3KV4xpRBuM6dHO1Lekb1LTj0S56xLq\ncPVlWcZwo+Z1rBDxBp4Vw6hg0oiGlzY48wH2BnW9+Zhvm+I6Vuuz2kwcQI02BknIPj2j3pCr\nOtL1wcqMZO5G2fEkXMeqYVyrL2vesGDEi+b1rIM3dE7v/gtjxFE/MdZSG/foticX9kjLu+rL\n6vLDN3dIy+q3otJ75T1oPKteceDanCZDVrkeOOuXrHxsavZPvm2KK++Ddo1r2fJnnwVNyPZO\naZfR44+VwcqMZJ5GceUd+LCJ3meGWLG3dFUh7hUCL64+85kSsY43xq8bgA9vN/5eiVi/KcTv\nsYAvMxaoEOvIWe+r2Ji4ALGAFiAW0ALEAlqAWEALEAtoAWIBLUAsoAWIBbQAsYAWIBbQAsQC\nWoBYQAsQC2gBYgEtQCygBYgFtACxgBYgFtACxAJagFhACxALaAFiAS1ALKAFiAW0ALGAFiAW\n0ALEAlr4fxB3ZpgYInc1AAAAAElFTkSuQmCC",
      "text/plain": [
       "Plot with title “Histogram of perm_diff”"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "obs_pct_diff <- 100*(200/23739 - 182/22588)\n",
    "conversion <- c(rep(0, 45945), rep(1, 382))\n",
    "perm_diff <- rep(0, 1000)\n",
    "for(i in 1:1000)\n",
    "    perm_diff[i] = 100*perm_func(conversion, 23739, 22588)\n",
    "hist(perm_diff, xlab = 'Conversion rate(percent)',\n",
    "                ylab = 'Frequence')\n",
    "abline(v = obs_pct_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:02:20.986897Z",
     "start_time": "2019-04-01T16:02:20.956Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "0.313"
      ],
      "text/latex": [
       "0.313"
      ],
      "text/markdown": [
       "0.313"
      ],
      "text/plain": [
       "[1] 0.313"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# P value\n",
    "mean(perm_diff > obs_pct_diff)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注意，由于抽样的随机性，前面对P值的计算会有所不同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:03:53.488931Z",
     "start_time": "2019-04-01T16:03:53.437Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "\t2-sample test for equality of proportions with continuity correction\n",
       "\n",
       "data:  c(200, 182) out of c(23739, 22588)\n",
       "X-squared = 0.14893, df = 1, p-value = 0.3498\n",
       "alternative hypothesis: greater\n",
       "95 percent confidence interval:\n",
       " -0.001057439  1.000000000\n",
       "sample estimates:\n",
       "     prop 1      prop 2 \n",
       "0.008424955 0.008057376 \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 内置的假设检验\n",
    "prop.test(x=c(200, 182), n=c(23739, 22588), alternative = \"greater\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### T Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:06:20.972203Z",
     "start_time": "2019-04-01T16:06:20.930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "\tWelch Two Sample t-test\n",
       "\n",
       "data:  Time by Page\n",
       "t = -1.0983, df = 27.693, p-value = 0.1408\n",
       "alternative hypothesis: true difference in means is less than 0\n",
       "95 percent confidence interval:\n",
       "      -Inf 0.1959674\n",
       "sample estimates:\n",
       "mean in group Page A mean in group Page B \n",
       "            1.263333             1.620000 \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t.test(Time~Page, data=session_times, alternative = 'less')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ANOVA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:12:51.149090Z",
     "start_time": "2019-04-01T16:12:51.040Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Page</th><th scope=col>Time</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>Page 1</td><td>164   </td></tr>\n",
       "\t<tr><td>Page 2</td><td>178   </td></tr>\n",
       "\t<tr><td>Page 3</td><td>175   </td></tr>\n",
       "\t<tr><td>Page 4</td><td>155   </td></tr>\n",
       "\t<tr><td>Page 1</td><td>172   </td></tr>\n",
       "\t<tr><td>Page 2</td><td>191   </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ll}\n",
       " Page & Time\\\\\n",
       "\\hline\n",
       "\t Page 1 & 164   \\\\\n",
       "\t Page 2 & 178   \\\\\n",
       "\t Page 3 & 175   \\\\\n",
       "\t Page 4 & 155   \\\\\n",
       "\t Page 1 & 172   \\\\\n",
       "\t Page 2 & 191   \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Page | Time | \n",
       "|---|---|---|---|---|---|\n",
       "| Page 1 | 164    | \n",
       "| Page 2 | 178    | \n",
       "| Page 3 | 175    | \n",
       "| Page 4 | 155    | \n",
       "| Page 1 | 172    | \n",
       "| Page 2 | 191    | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Page   Time\n",
       "1 Page 1 164 \n",
       "2 Page 2 178 \n",
       "3 Page 3 175 \n",
       "4 Page 4 155 \n",
       "5 Page 1 172 \n",
       "6 Page 2 191 "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "library(lmPerm)\n",
    "four_sessions <- read.csv(\"../psds_data/four_sessions.csv\")\n",
    "head(four_sessions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:11:07.153605Z",
     "start_time": "2019-04-01T16:11:07.104Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"Settings:  unique SS \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Component 1 :\n",
       "            Df R Sum Sq R Mean Sq Iter Pr(Prob)  \n",
       "Page         3    831.4    277.13 4477  0.09002 .\n",
       "Residuals   16   1618.4    101.15                \n",
       "---\n",
       "Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "summary(aovp(Time~Page, data=four_sessions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:12:06.467034Z",
     "start_time": "2019-04-01T16:12:06.432Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "            Df Sum Sq Mean Sq F value Pr(>F)  \n",
       "Page         3  831.4   277.1    2.74 0.0776 .\n",
       "Residuals   16 1618.4   101.2                 \n",
       "---\n",
       "Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# R built-in anova\n",
    "summary(aov(Time~Page, data=four_sessions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Chi-Square Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:16:24.574240Z",
     "start_time": "2019-04-01T16:16:24.531Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Headline</th><th scope=col>Click</th><th scope=col>Rate</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>Headline A</td><td>Click     </td><td> 14       </td></tr>\n",
       "\t<tr><td>Headline A</td><td>No-click  </td><td>986       </td></tr>\n",
       "\t<tr><td>Headline B</td><td>Click     </td><td>  8       </td></tr>\n",
       "\t<tr><td>Headline B</td><td>No-click  </td><td>992       </td></tr>\n",
       "\t<tr><td>Headline C</td><td>Click     </td><td> 12       </td></tr>\n",
       "\t<tr><td>Headline C</td><td>No-click  </td><td>988       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lll}\n",
       " Headline & Click & Rate\\\\\n",
       "\\hline\n",
       "\t Headline A & Click      &  14       \\\\\n",
       "\t Headline A & No-click   & 986       \\\\\n",
       "\t Headline B & Click      &   8       \\\\\n",
       "\t Headline B & No-click   & 992       \\\\\n",
       "\t Headline C & Click      &  12       \\\\\n",
       "\t Headline C & No-click   & 988       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Headline | Click | Rate | \n",
       "|---|---|---|---|---|---|\n",
       "| Headline A | Click      |  14        | \n",
       "| Headline A | No-click   | 986        | \n",
       "| Headline B | Click      |   8        | \n",
       "| Headline B | No-click   | 992        | \n",
       "| Headline C | Click      |  12        | \n",
       "| Headline C | No-click   | 988        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Headline   Click    Rate\n",
       "1 Headline A Click     14 \n",
       "2 Headline A No-click 986 \n",
       "3 Headline B Click      8 \n",
       "4 Headline B No-click 992 \n",
       "5 Headline C Click     12 \n",
       "6 Headline C No-click 988 "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "click_rate <- read.csv('../psds_data/click_rates.csv')\n",
    "head(click_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:16:27.009297Z",
     "start_time": "2019-04-01T16:16:26.976Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Click No-click\n",
      "Headline A    14      986\n",
      "Headline B     8      992\n",
      "Headline C    12      988\n"
     ]
    }
   ],
   "source": [
    "clicks <- matrix(click_rate$Rate, nrow=3, ncol=2, byrow=TRUE)\n",
    "dimnames(clicks) <- list(unique(click_rate$Headline), unique(click_rate$Click))\n",
    "print(clicks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:17:14.389442Z",
     "start_time": "2019-04-01T16:17:14.346Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "\tPearson's Chi-squared test with simulated p-value (based on 2000\n",
       "\treplicates)\n",
       "\n",
       "data:  clicks\n",
       "X-squared = 1.6659, df = NA, p-value = 0.4893\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "chisq.test(clicks, simulate.p.value = TRUE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:17:24.176148Z",
     "start_time": "2019-04-01T16:17:24.147Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "\tPearson's Chi-squared test\n",
       "\n",
       "data:  clicks\n",
       "X-squared = 1.6659, df = 2, p-value = 0.4348\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "chisq.test(clicks, simulate.p.value = FALSE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fisher's Exact Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T16:18:00.723617Z",
     "start_time": "2019-04-01T16:18:00.697Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "\tFisher's Exact Test for Count Data\n",
       "\n",
       "data:  clicks\n",
       "p-value = 0.4824\n",
       "alternative hypothesis: two.sided\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fisher.test(clicks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R 3.4",
   "language": "R",
   "name": "ir34"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.5.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "273.2px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
